#!/usr/bin/env python

#Copyright (C) 2008 Luis Felipe Millani (lf.millani@gmail.com)

#This file is part of MagicLyrics.

#MagicLyrics is free software: you can redistribute it and/or modify
#it under the terms of the GNU General Public License as published by
#the Free Software Foundation, either version 3 of the License, or
#(at your option) any later version.

#MagicLyrics is distributed in the hope that it will be useful,
#but WITHOUT ANY WARRANTY; without even the implied warranty of
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#GNU General Public License for more details.

#You should have received a copy of the GNU General Public License
#along with MagicLyrics.  If not, see <http://www.gnu.org/licenses/>.
def get_text_from_page(source):
	"""
	get_text_from_page(source) -> string
	
	given the html code of a webpage, remove everything which the user
	wouldn't see when acessing the page in a browser
	currently it also removes blank lines from the page, but in the
	future this may be changed
	"""
	def ascii_code_to_ascii(text):
		text.replace('a','b')
		alt={'&quot;':'"','&amp;':'&','&lt;':'<','&gt;':'>','&nbsp;':' ','&shy;':'&#173;'}
		for i in alt:
			text=text.replace(i,alt[i])
		for i in range(0,10):
			text=text.replace('&#%d;' % i,chr(i))
		for i in range(0,100):
			text=text.replace('&#%02d;' % i,chr(i))
		for i in range(0,128):#183):
			text=text.replace('&#%03d;' % i,chr(i))
		return text
	
	if source.find('<body')>-1:
		source=source[source.find('<body'):source.rfind('</body>')].replace('\n','').replace('\t','')
		source=source[source.find('>')+1:]
	source=source.replace('\n','')
	text=''
	
	
	source_lower=source.lower()
	while source_lower.find('<')>-1:
		pos=source_lower.find('<')
		text+=source[:pos]
		if pos in (source_lower.find('</h1>'),source_lower.find('</h2>'),source_lower.find('</h3>'),source_lower.find('</h4>'),source_lower.find('</h5>'),source_lower.find('</h6>'),source_lower.find('<p>'), source_lower.find('<div>'), source_lower.find('</div>'), source_lower.find('<br>'), source_lower.find('<br/>'), source_lower.find('<br />'), source_lower.find('< br>'), source_lower.find('< br/>'), source_lower.find('< br />')):
			text+='\n'
		if pos==source_lower.find('<script'):
			left=source_lower.find('</script>')+len('</script>')
			source=source[left:]
			source_lower=source_lower[left:]
		elif pos in (source_lower.find('<a '),source_lower.find('<a>')): # just '<a' wouln't work as wanted
			left=source_lower.find('</a>')+len('</a>')
			source=source[left:]
			source_lower=source_lower[left:]
		elif source_lower.find('>')>-1:
			inside_quotes=True
			while inside_quotes:
				inside_quotes=False
				while -1<source.find('"')<source.find('>'):
					inside_quotes=not inside_quotes
					source=source[source.find('"')+1:]
				if inside_quotes:
					source=source[source.find('"')+1:]
			source=source[source.find('>')+1:]
			source_lower=source.lower()
		else:
			source_lower=''
	text=ascii_code_to_ascii(text)
	
	
	
	# remove blank lines
	# this has the bad side effect of making the lyrics look ugly
	tmp=text.splitlines()
	text=''
	for line in tmp:
		if not line.replace(' ','').replace('\t','')=='':
			text+=line+'\n'#.strip(' ')+'\n'
	text=text[:-1] # remove trailing \n
	
	
	return text#.replace('  ',' ')#.replace('\'','')